'''
Parse html files downloaded from LFC History to Liverbird database
'''

from html.parser import HTMLParser
import re

if __name__ == '__main__':
    pass

class Download_Games():
    def __init__ (self, id_range):
        self.range = id_range # range(8037,9319)
        self.base_url =  'http://www.lfchistory.net/SeasonArchive/Game/' #'http://www.lfchistory.net/SeasonArchive/FriendlyGame/'
    
    def create_files(self, path):
        import urllib.request
        
        for id in self.range:
            in_path = self.base_url+ str(id)
            out_path = '{0}/game_{1}.txt'.format(path, id)
            print (id, in_path, out_path)
            
            try:
                data = urllib.request.urlopen(in_path).read()
            except:
                print ("no game ")
                continue
            
            with open(out_path, mode='w', encoding='utf-8') as out_file:
                out_file.write(data.decode('utf-8'))


class MyHTMLParser(HTMLParser):
    def handle_starttag(self, tag, attrs):
        #print("Encountered a {} start tag".format(tag))
      
        if tag == "h2":
            self.record = []
        
        self.key = tag
            
 
    def handle_data(self, attr):
        attr = attr.rstrip()
        if len(attr) < 2:
            return
        
        if self.key == "td":
            self.record.append(["Params", attr])
    
        if self.key == "h3":
            if not (attr == "Goals" or attr == "Notes"):
                self.record.append (["Result", attr])
 
        if self.key == "a":
            self.record.append (["Goals", attr])
        
        if self.key == "span":
            self.record.append (["Notes", attr])

from parse_games_new import Parse_Recent_Games

class Parse_Friendlies(Parse_Recent_Games): 
        
    def split_params(self, line):
        item = line.split(',')
        assert item[0] == "\'Params\'"
        return item[1][2:-2]
               
    
    def parse_score_x(self, line):
        s = line.split(',')
        assert s[0] == "\'Result\'"
        self.parse_score(s[1][2:-2])
        #print (self.dict['opposition'], self.dict['score'])
        
    def parse_date_x(self, line):
        self.parse_date(self.split_params(line))
        #print (self.dict['date'])
        
    def parse_stadium_x(self, line):
        self.parse_stadium(self.split_params(line))
        #print (self.dict['stadium'])

    def parse_tournament_x(self, line):
        self.dict['notes'].append('Турнир: ' + self.split_params(line))
        #print (self.dict['notes'])

    def parse_attendace(self, line):
        item = line.split(',',1)
        
        assert item[0] == "\'Params\'"
        self.dict['attendance'] = item[1][2:-4].replace(',', '')
        
        if self.dict['attendance'] == 'Unknown':
            self.dict['attendance'] = 'NULL'
        
        #print (self.dict['attendance'])

    def parse_notes(self, line):
        if line.find('Notes') > 0:

            note = 'Примечания: '
            
            for item in line.split('['):
                if item.find('Notes') > 0:
                    string = ' ' + item.split(',',1)[1][2:-4]
                    note += string   

            self.dict['notes'].append(note)
        
        #print (self.dict['notes'])
           
    def parse_goals(self, line):
        if line.find('Goals') > 0:
            self.dict['goals'] = 'Голы '
        
            for item in line.split('['):
                if item.find('Goals') > 0:
                    string = ' ' + item.split(',',1)[1][2:-4]
                    if (string.find('\'')  > 0 and len(string) < 7) or \
                       (string.find('pen') > 0 and len(string) < 7):
                        string = '(' + string[2:] + ')' 
               
                    self.dict['goals'] += string

         #   print (self.dict['goals'])

    def process_recent(self, path):
        self.dump = open(path, encoding='utf-8').readlines()
        self.all_dicts = {} 
        
        for line in self.dump:
            self.dict = {}
            self.dict['notes'] = []
            
            items = line.split('[')
          
            ''' Parse opposition, score, date, stadium, tournament, attendance, notes, goals '''
            self.parse_score_x(items[2])
            self.parse_date_x(items[3])
            self.parse_stadium_x(items[4])
            self.parse_tournament_x(items[5])
            self.parse_attendace(items[6])
            self.parse_notes(line)
            self.parse_goals(line)
          
            self.all_dicts[items[0][:-1]] = self.dict
            #print ('\n')
        
        #print (self.all_dicts, sep = '\n')

    def find_missing(self, path, keyname):
        
        lvbd_item = []
        
        with open(path, 'r', encoding = 'utf-8') as stad_file:
            for line in stad_file.readlines():
                 a = line.rstrip().split('\t', 1)
                 
                 lvbd_item.append([a[0], a[1]])

        missing = []
        
        for key in self.all_dicts.keys():
            for item in lvbd_item:
                if item[1].find(self.all_dicts[key][keyname]) >= 0:
                    break
            else:
                missing.append(self.all_dicts[key][keyname])
        
        print('Missing {0} elements: {1}'.format(len(set(missing)), set(missing)))

        
        
class Parse_Friendlies_old():    
    def __init__ (self):
        pass
    
    def read_file(self, path):
        import os
        dump = open("dump.txt", 'w')
 
        for infile in os.listdir(path):
            print ("current file is: ", infile)
            
            parser = MyHTMLParser()
            
            data = open('out/'+infile,'r').read()
            
            start = data.find("<h2 class=\"page-name\">Report</h2>")
            
            end = data.find("<div class=\"clearfix\"></div>")
            
            try:
                parser.feed(data[start:end])
            except:
                print ("Exception!")
            
            dump.write (str(infile[5:].split('.')[0]) + ',' + str(parser.record) + '\n')
   
    def parse_score(self, line):
        i2 = line.split(',')
        assert i2[0] == "\'Result\'"
            
        score_parts = i2[1][1:-1].split('-')
            
        pattern = re.compile(r'[\'\"]([\w\s\'-]*)\s(\d{1,2}) - (\d{1,2})\s([\w\s]*)[\'\"]$') 
            
        try:
            parsed = pattern.search(i2[1][:-1]).groups()
        except:
            print(line)
            raise ValueError("Wrong Score 1")
            
        if parsed[0] == "Liverpool":
            we_score = parsed[1]
            they_score = parsed[2]
            opposition = parsed[3]
        elif parsed[3] == "Liverpool":
            opposition = parsed[0]
            they_score = parsed[1]
            we_score = parsed[2]
        else: 
            print(parsed)
            raise ValueError("Wrong Score 2")
            
        self.dict['opposition'] = opposition
        self.dict['score'] = "{0}:{1}".format(we_score, they_score) 
           
    def parse_date(self, line):
        item = line.split(',')
        
        assert item[0] == "\'Params\'"
               
        date_parts = item[1][2:-2].split('.', 2)
        self.dict['date'] = '{0}-{1}-{2}:T00:00:00'.format(date_parts[2],date_parts[1],date_parts[0])
        
    
    def parse_stadium(self, line):
        item = line.split(',')
        
        assert item[0] == "\'Params\'"
        self.dict['stadium'] = item[1][2:-2]
   
    def parse_tournament(self, line):
        item = line.split(',')
        
        assert item[0] == "\'Params\'"
        self.dict['tournament'] = item[1][2:-2]     

    def parse_attendace(self, line):
        item = line.split(',',1)
        
        assert item[0] == "\'Params\'"
        self.dict['attendance'] = item[1][2:-4]
        
        if self.dict['attendance'] == 'Unknown':
            self.dict['attendance'] = ''
 
    def parse_goals(self, line):
        if line.find('Goals') > 0:
            self.dict['goals'] = 'Голы '
        else:
            self.dict['goals'] = ''
        
        for item in line.split('['):
            if item.find('Goals') > 0:
                string = ' ' + item.split(',',1)[1][2:-4]
                if (string.find('\'')  > 0 and len(string) < 7) or \
                   (string.find('pen') > 0 and len(string) < 7):
                    string = '(' + string[2:] + ')' 
           
                self.dict['goals'] += string

    def parse_notes(self, line):
        
        if line.find('Notes') > 0:
            self.dict['notes'] = 'Примечания '
        else:
            self.dict['notes'] = ''

        for item in line.split('['):
            if item.find('Notes') > 0:
                string = ' ' + item.split(',',1)[1][2:-4]
                self.dict['notes'] += string   

    def read_dump(self, path):
        self.dump = open(path, encoding='utf-8').readlines()
        self.all_dicts = {} 
        
        for line in self.dump:
            self.dict = {}
            
            items = line.split('[')
          
            ''' Parse opposition, score, date, stadium, tournament, attendance, notes, goals '''
            self.parse_score(items[2])
            self.parse_date(items[3])
            self.parse_stadium(items[4])
            self.parse_tournament(items[5])
            self.parse_attendace(items[6])
            self.parse_notes(line)
            self.parse_goals(line)
          
            self.all_dicts[items[0][:-1]] = self.dict

        
#        print (self.all_dicts, sep = '\n')
    def write_all_dict(self, dest):
        with open(dest,'w', encoding = 'utf-8') as out_file:
            for key in self.all_dicts.keys():
                
                out_file.write('\'{0}\', \'{1}\',  \'{2}\', \'{3}\', \'{4}\', \'{5}\', \' Турнир:{6} {7} {8}\',\n'.format(\
                                                            key, \
                                                            self.all_dicts[key]['opposition'],\
                                                            self.all_dicts[key]['score'],
                                                            self.all_dicts[key]['date'],
                                                            self.all_dicts[key]['stadium'],
                                                            self.all_dicts[key]['attendance'],
                                                            self.all_dicts[key]['tournament'],
                                                            self.all_dicts[key]['notes'],
                                                            self.all_dicts[key]['goals'])) 

    def find_missing(self, path, keyname):
        
        lvbd_item = []
        
        with open(path, 'r', encoding = 'utf-8') as stad_file:
            for line in stad_file.readlines():
                 a = line.rstrip().split('\t', 1)
                 
                 lvbd_item.append([a[0], a[1]])

        missing = []
        
        for key in self.all_dicts.keys():
            for item in lvbd_item:
                if item[1].find(self.all_dicts[key][keyname]) >= 0:
                    break
            else:
                missing.append(self.all_dicts[key][keyname])
        
        print('Missing {0} elements: {1}'.format(len(set(missing)), set(missing)))
        
        
pf = Parse_Friendlies()

pf.process_recent('data/friendlies_dump.txt')

pf.find_missing('data/stadium.txt', 'stadium')
pf.find_missing('data/opposition.txt', 'opposition')

print ("Done")
